import scrapy
from movie.items import MovieItem

class MeijuSpider(scrapy.Spider):
    name = "meiju"
    allowed_domains = ["https://www.meijutt.net"]
    start_urls = ['https://www.meijutt.net/topiclist/2024xinjutop.html','https://www.meijutt.net/']

    def parse(self, response):
        for item in response.xpath('//div[@class="list3_cn_box "]/div[@class="topic_box"]'):
            movie_item = MovieItem()
            moive_name = item.xpath('.//ul[@class="topic_con"]/li[1]/a/text()').extract()
            moive_href = item.xpath('.//ul[@class="topic_con"]/li[1]/a/@href').extract()
            if moive_name and moive_href:
                movie_item['moive_name'] = moive_name
                movie_item['moive_href'] = moive_href
                yield movie_item
            else:
                self.logger.warning(f"Missing data for item: {item.get()}")

        for i in range(2, 9):
            url = 'https://www.meijutt.net/topiclist/2024xinjutop-{}.html'.format(str(i))
            yield scrapy.Request(url)
